﻿#include "ASTTokenizer.h"
using namespace yzrilyzr_lang;
namespace yzrilyzr_ast{
	std::shared_ptr<TokenReader> ASTTokenizer::tokenize(const String & s){
		std::shared_ptr<TokenReader> ptokens=std::make_shared<TokenReader>();
		TokenReader & tokens=*ptokens;
		const char * chars=s.c_str();
		CharReader reader(chars);
		while(reader.has()){
			char c=reader.get();
			if((c < '0' || c>'9') && isWord(c)){
				std::stringstream token;
				while(reader.has() && isWord(c=reader.get())){
					token << c;
					reader.next();
				}
				tokens.add(reader, TokenType::ID, std::string(token.str()));
			} else if(c >= '0' && c <= '9' || (c == '.' && reader.get(1) >= '0' && reader.get(1) <= '9')){
				std::stringstream token;
				while(reader.has() && isNumber(c=reader.get())){
					token << c;
					reader.next();
				}
				tokens.add(reader, TokenType::Number, std::string(token.str()));
			} else if(c == '"'){
				std::stringstream token;
				reader.next();
				while(reader.has()){
					c=reader.get();
					if(c == '"' && reader.get(-1) != '\\') break;
					token << c;
					reader.next();
				}
				reader.next();
				tokens.add(reader, TokenType::StringType, std::string(token.str()));
			} else if(c == '`'){
				std::stringstream token;
				reader.next();
				while(reader.has()){
					c=reader.get();
					if(c == '`' && reader.get(-1) != '\\') break;
					token << c;
					reader.next();
				}
				reader.next();
				tokens.add(reader, TokenType::StringType, std::string(token.str()));
			} else if(c == '\''){
				std::stringstream token;
				reader.next();
				while(reader.has()){
					c=reader.get();
					if(c == '\'' && reader.get(-1) != '\\') break;
					token << c;
					reader.next();
				}
				reader.next();
				tokens.add(reader, TokenType::StringType, std::string(token.str()));
			} else if(c == '/' && reader.get(1) == '/'){
				reader.next();
				reader.next();
				std::stringstream token;
				while(reader.has()){
					c=reader.get();
					if(c == '\n') break;
					token << c;
					reader.next();
				}
				reader.next();
				tokens.add(reader, TokenType::LineComment, std::string(token.str()));
			} else if(tokens.size() > 0 &&
					  tokens.getToken(tokens.size() - 1).type != TokenType::ID &&
					  c == '-' &&
					  reader.get(1) == '-'){
				reader.next();
				reader.next();
				std::stringstream token;
				while(reader.has()){
					c=reader.get();
					if(c == '\n') break;
					token << c;
					reader.next();
				}
				reader.next();
				tokens.add(reader, TokenType::LineComment, std::string(token.str()));
			} else if(c == '/' && reader.get(1) == '*'){
				reader.next();
				reader.next();
				std::stringstream token;
				while(reader.has()){
					c=reader.get();
					if(c == '/' && reader.get(-1) == '*') break;
					token << c;
					reader.next();
				}
				reader.next();
				auto a=token.str();
				tokens.add(reader, TokenType::BlockComment, std::string(a.c_str(), 0, a.length() - 1));
			} else{
				/*for(char a:getSkip()){
					if(c==a){
						reader.next();
						goto lb1;
					}
				}*/
				tokens.add(reader, TokenType::Symbol, readSymbol(reader));
				/*lb1:
				{}*/
			}
		}
		return ptokens;
	}
}